import jdk.internal.util.xml.impl.Input;
import lombok.SneakyThrows;
import org.ansj.domain.Term;
import org.ansj.splitWord.analysis.ToAnalysis;

import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.util.HashSet;
import java.util.Scanner;
import java.util.Set;
import java.util.stream.Collectors;

/**
 * @author yuisama
 * @date 2023/04/17 21:03
 **/

public class Main {
    @SneakyThrows
    public static void main(String[] args) {
        File file = new File("D:\\代码保存\\搜索引擎\\docs\\api\\compact2-summary.html");

        try(InputStream is = new FileInputStream(file)){
            try(Scanner scanner = new Scanner(is,"ISO-8859-1")){
                StringBuilder sb = new StringBuilder();

                while(scanner.hasNextLine()){
                    String line = scanner.nextLine();
                    line = line.replaceAll("<script[^>]*>[^<]*</script>"," ");
                    line = line.replaceAll("<[^>]*>"," ");
                    sb.append(line).append(" ");
                }
                String content = sb.toString().replaceAll("\\s+"," ").trim();

                System.out.println(ToAnalysis.parse(content).getTerms().size());


//                Set<Term> set = new HashSet<>(ToAnalysis.parse(content).getTerms());



                Set<String> set = ToAnalysis.parse(content)
                        .getTerms()
                        .stream()
                        .map(Term::getName)
                        .collect(Collectors.toSet());
                System.out.println(set.size());
            }
        }
    }
}

